package org.NooLab.itexx.retrieval.google;


/**
 * This implementation class for pre- and postprocessing of google result pages
 * need not to know anything about the "content" of the interfaces (their implementation);
 * 
 * we achieved that by cross-linking interfaces between provider and consumer;
 * such it is a plugin
 * 
 */

import java.io.IOException;
import java.util.ArrayList;


import org.NooLab.itexx.retrieval.PageIntf;
import org.NooLab.itexx.retrieval.intf.DocumentsPPPIntf;
import org.NooLab.itexx.retrieval.intf.PPPConsumerIntf;
import org.NooLab.utilities.files.DFutils;
import org.NooLab.utilities.logging.PrintLog;
import org.NooLab.utilities.resources.ResourceLoader;
import org.NooLab.utilities.strings.ArrUtilities;
import org.NooLab.utilities.strings.StringsUtil;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;




/**
 * 
 * this class performs operations on the result page of a google query, such that  <br/>
 * - links to cache entries are removed,
 * - top and bottom of page, command bar links are removed,
 * 
 * remove page translation links   <a href="http://translate.google.ch/translate
 * 
 */
public class GooglePreProc implements DocumentsPPPIntf{
	
	String defaultHTMLOpening = "<!doctype html> <head><title> $$$ </title></head><body>";
	// =================================
	
	// object references ..............
	

	// main variables / properties ....

	
	// constants ......................
	
	
	// volatile variables .............

	
	// helper objects .................
	PrintLog out  ;
	StringsUtil strgutil = new StringsUtil();
	
	// ========================================================================
	public GooglePreProc(){
		
	}
	public GooglePreProc( PPPConsumerIntf consumer ){
		
		// this should send an anchor to the main class , such that this preproc routine
		// can be executed over there
		
		if (consumer!=null){
			consumer.addPreProcessor( this ) ;
		}
		
	}
	// ========================================================================
	
	/*
	      
	
	*/
	 
	public void performPPP( PageIntf page) {
		String str, pageAddress, htmlContent = "" ;
		
		try{
			
			pageAddress = page.getWebURL().getURL() ;

			// just for DEBUG page.setText( (String) ("<!--  abc -->" + page.getText()) ) ;
			
			if (( pageAddress.toLowerCase().contains("www.google.")==false) || 
				(page.getContentType().contains("text")==false )){
				return ;
			}
			
			
			str = page.getHTML() ;
			
			// do the adjustments here
			// we remove the stuff we do not need : headers, finals, href attributes
			// then re-assembling a valid HTML doc again, which can be parsed
			
			str = StringUtils.substringBetween( htmlContent, "<title>","</title>") ;
			
			// put the changes back to the page :: object
			
			System.out.println(" google preproc applied...");
			page.setHTML(str) ;
			
			
		}catch(Exception e){
			e.printStackTrace() ;
		}	
		
		
	}
	
	public String performPPP(String htmltext) {
		String str, str2,htmlstr="";
		int p1,p2;
		
		try{
			
			str = htmltext ;
			
			p1 = str.indexOf("<body>");
			p2 = str.indexOf("</body>");
			
			str = str.substring(p1, p2) ;
			str = "<html>\n"+str+"\n</html>";
			
			p1 = str.indexOf("<script>");
			p2 = str.indexOf("</script>");
			
			
			
			while (p1>0){
				
				if ((p1>0) && (p2>p1)){
					str2 = str.substring(p2+9,str.length());
					str = str.substring(0,p1)+ str2 ; 
				}	
			}
			
			htmlstr=str;
			
		}catch(Exception e){
		}
		return htmlstr;
	}
	
	
	
	public void reduceRawGoogleHtml(PageIntf page) {
		// 
		// <div role="main" id="res" class="med">
		// <div class="med" id="res" role="main">  
		
		//   <div id="search"><h2 class="hd">
		
		
		StringsUtil strgutil = new StringsUtil();
		int p,pn,pc,mainStartPos=-1;
		String htmlstr, ds ;
		int[] divClose ,divpositions ;
		
		htmlstr =  page.getHTML() ;
		
		if (htmlstr.contains("\"main\"")==false){
			return;
		}
		
		divpositions = strgutil.indexesOfStrings(htmlstr, new String[]{"<div"}, 0, true);
		
		for (int i=0;i<divpositions.length;i++){
			
			p = divpositions[i] ;
			pc = htmlstr.indexOf(">", p+1);
			pn = htmlstr.indexOf("<", p+2);
			
			if ((p>0) && (pc>p) && (pc-p<40) && (pc<pn)){
				ds = htmlstr.substring(p,pc) ;
				//if ((ds.contains("\"main\""))) {
				if ((ds.contains("id=\"search\""))) {
					
					mainStartPos = p;
					pc = htmlstr.indexOf("</body>");
					htmlstr = htmlstr.substring(mainStartPos, pc+7);
					
					break;
				}
			}
			
			
		}// i->
		if (mainStartPos<0){return;}
		divpositions = strgutil.indexesOfStrings(htmlstr, new String[]{"<div"}, 0, true);
		divClose = strgutil.indexesOfStrings(htmlstr, new String[]{"</div>"}, 0, true);
		
		ArrayList<Integer> d1 = new ArrayList<Integer> (); 
		ArrayList<Integer> d2 = new ArrayList<Integer> (); 
		
		d1 = ArrUtilities.changeArraystyle(divpositions) ;
		d2 = ArrUtilities.changeArraystyle(divClose) ;
		
		while (d1.size()<d2.size()){
			d2.remove(d2.size()-1) ;
		}
		
		pc=0;
		if (d2.size()>0){
			pc = d2.get(d2.size()-1) ;
		}
		if (pc>0){
			htmlstr = htmlstr .substring(0, pc+6); 
		}
		// body present ?
		
		if (htmlstr.indexOf("</body>")<0){
			htmlstr = htmlstr +"</body>";
		}
		htmlstr = "<html>"+htmlstr +"</html>";
		
		// removing some divs
		// <div style="position: absolute; top: 1px; width: 132px;" onclick="google.srp&&google.srp.qs(event)" role="navigation" id="leftnav">
		
		// removing all scripts
		
		
		page.setHTML(htmlstr) ;
	}
	
	/**
	 * 
	 * note that there is a dependency of the output from the user-agent !!!
	 * google returns a old-style table or div stuff
	 * 
	 * <FORM method=get name=gs action=/search> <INPUT title=Suche name=q maxLength=2048 value="putnam program" size=41><INPUT name=num value=100 type=hidden><INPUT name=hl value=de type=hidden><INPUT name=newwindow value=0 type=hidden><INPUT name=sa value=N type=hidden><FONT size=-1>&nbsp;</FONT><INPUT name=btnG value=Suche type=submit><FONT size=-1>&nbsp;</FONT></FORM></
	 * 
	 * 
	 * @param page
	 * @return
	 * @throws IOException 
	 */
	public String selectSearchResults(PageIntf page) throws IOException {
	
		StringsUtil strgutil = new StringsUtil();
	
		boolean detected=false;
		String[] strs = new String[0];
		
		String cdText="",html;
		Element contentDiv;
		int k=0;
		
		html = page.getHTML();
		
		// html = strgutil.removeStringsBetweenEmbracingPairs( html, "<style", "</style>", -1);
		// html = strgutil.removeStringsBetweenEmbracingPairs( html, "<style ", "/>", -1);
		
		// <B>abc</B>
		int z=0;
		
		html = replaceSimpleTags(html, new String[]{"em","b","i"});
		
		// ><div id="search"><!--a--><h2 class="hd">
		Document soupdoc = Jsoup.parse(html) ;

		// the problem is, that google delivers malformed html... without " for the id
		// such, jsoup does NOT recognize !!!
		boolean containsIdSearch;   
		
		
		removeScripts(soupdoc);
		
		removeCacheLinks(soupdoc);
		
		removeGooglePlus(soupdoc) ;
		
		html = strgutil.replaceAll( soupdoc.html(), "<div></div>","");
		soupdoc = Jsoup.parse(html) ;
		
		// this must be AFTER re-establishing the html string
		int n = enumerateItems(soupdoc);
		
		
		containsIdSearch = soupdoc.select("div#search").first() != null;
		if (containsIdSearch==false){
			containsIdSearch = soupdoc.select("DIV#search").first() != null;
		}
		if (containsIdSearch==false){
			containsIdSearch = html.indexOf("<div id=\"search\">") > 0;
		}

		if (containsIdSearch==false){
			containsIdSearch = html.toLowerCase().contains("<div id=search>") ;			
		}

		boolean containsCommentAtStart = html.indexOf("<!--a-->") > 0;
		
		String allScripts = getAllScripts(soupdoc);
		
		if ((containsIdSearch) || (containsCommentAtStart)){
			soupdoc = assemblingBareList(soupdoc,allScripts) ;
		}
		
		detected = containsIdSearch;
		
		String tmpFile = DFutils.getTempFile("/googlehtml-1.txt");
		// (new DFutils()).writeFileSimple( tmpFile, html) ;
		
		// check whether we are already logged into google!
		Element div = soupdoc.select("div#gbqfqw").first() ; 
		boolean loggedIn = false;// (div==null);
		
		if (detected==false){
			
			boolean rB ;
			rB = html.indexOf("<DIV id=pocs0>")>0;
			if (rB){
				// html = (new DFutils()).readFile2String(tmpFile );
				html = html.replace("<body", "<!-- i -->");
				html = html.replace("<BODY", "<!-- i -->");
				soupdoc = Jsoup.parse(html);
				if (loggedIn==false){
					html = prepareLoginAdvice(soupdoc);	
				}
			}else{
				html = prepareBrowserAdvice();
			}
		}else{
			html = soupdoc.html() ;
		}
		
		html = replaceSimpleTags(html, new String[]{"em","i"});
		
		page.setHTML(html) ;
		return html;
	}
	
	
	private int enumerateItems(Document soupdoc) {
		
		// <li class="g"> 
		
		Elements listItems = soupdoc.select("li.g") ;
		int count = listItems.size() ;
		String str,html = soupdoc.html();
		
		int z=1;
		int p2=0,p3,p1=0;
		
		while ((p1>=0) && (p2>=0)){
			
			p1 = html.indexOf("<li",p2);
			
			if (p1>0){
				p2 = html.indexOf( "class",p1+1);
				if (p2<0){
					break;
				}
				p3 = html.indexOf( ">",p2+1); if (p3<0)p3=html.length();
				str = html.substring(p2, p3).replace("\"", "");
				if (str.contentEquals("class=g")){
					String lic = " id=\"itx-cnt\" name=\""+z+"\" "; z++;
					lic = lic + " style=\"padding-top:20px;margin-top:20px;\" ";
					html = strgutil.insert(html, lic, p3);
					
					str = html.substring(p1-3, p3+250);
					str = str+ " " ;
					int n=str.length();
				}
			}
			
		}// -> p1,p2
		
		html = strgutil.replaceAll(html, "\n\n", "\n ");
		html = strgutil.replaceAll(html, "\n\n", "\n ");
		html = strgutil.replaceAll(html, "  ", " ");
		html = strgutil.replaceAll(html, "  ", " ");
		
		// html = strgutil.replaceAll( html, "<div></div>","");
		soupdoc = Jsoup.parse(html) ;
		
		
		return count;
	}
	
	
	private String replaceSimpleTags(String html, String[] stags) {
		// 
		int z;
		String tag;
		
		for (int i=0;i<stags.length;i++){
			tag = stags[i];
			tag = "<"+tag+">";
			html = strgutil.replaceAll(html, tag, "");
			
			tag = stags[i];
			tag = "</"+tag+">";
			html = strgutil.replaceAll(html, tag, "");	

		}// i->
		
		return html;
	}
	
	// FIXME put this into a JsoupHandler class 
	private void removeGooglePlus(Document soupdoc) {
		// 

		// any div containing in 
		// - ownText() "Google+",  
		// - google.x in outerHTML (actually, in the script)
		
		// class="pplsrsl
		Elements gplusSpan = soupdoc.select("span.pplsrsl") ;
		Elements gplusDivs  = soupdoc.select("div.pplsrsl") ;

		removeElements( soupdoc, gplusSpan ,0 );
		removeElements( soupdoc, gplusDivs ,0 );
		
		gplusDivs = soupdoc.select("span.esc slp") ;
		removeElements( soupdoc, gplusDivs ,0 );
		
		gplusDivs = soupdoc.select("div.pplsrslc") ;
		removeElements( soupdoc, gplusDivs ,0 );
			
	}
	private void removeElements( Document soupdoc, Elements elements, int includeReparsing) {
		// 
		Element hele ;
		
		for (int i=0;i<elements.size();i++){
			hele = elements.get(i);
			hele.replaceWith(new Element(Tag.valueOf("div"), ""));
		}

		if (includeReparsing>0){
			String html = strgutil.replaceAll( soupdoc.html(), "<div></div>","");
			soupdoc = Jsoup.parse(html) ;
		}
	}
	
	private void removeCacheLinks(Document soupdoc) {
		/*
		 *  all spans of ::  <span class="vshid">
		 *      <a onmousedown="return rwt(this,'','','','8','AFQjCNEL6zqjrubVbDBDiCLZzwPbQGaN-Q','LJIZkVhXxc7o3Wv9Qcy-fw','0CGoQIDAH',null,event)" 
		 *         href="http://webcache.googleusercontent.com/search?q=cac
		 *         
		 *  we also could check for "webcache.googleusercontent"
		 */
		
		Elements spanCacheLinks = soupdoc.select("span.vshid") ;
		removeElements( soupdoc, spanCacheLinks ,0 );
		
		Elements cites = soupdoc.select("cite") ;
		removeElements( soupdoc, cites ,0 );
	}
	
	private String prepareBrowserAdvice() {
		String html;
		html = "<html><meta creator=itexx></meta><body><p style=\"margin-top:40px;margin-left:20px;font-size:1.6em;\">"+
			   "Please install a compatible browser or update your installed one !</p>"+
			   "<p style=\"font-size:1.1em;margin-left:20px;\">(Internet Explorer on Windows, or Firefox on OSX or Linux)</p>"+
			   "<p style=\"margin-top:40px;margin-left:20px;font-size:1.3em;\">"+
			   "iTexx won't work if neither of these browsers is available on your system.</br>"+
			   "</p>"+
			   "<p style=\"margin-top:40px;margin-left:20px;font-size:1.2em;\">"+
			   "You should see something like the following output,</br> i.e. a processed list:</br></br>"+
			   "<img src=\"\"  />"+
			   "</p>"+
			   "</body></html>";
		return html;
	}
	private String prepareLoginAdvice(Document soupdoc) throws IOException {
		// 
		String html,xhtml,str,imagesource="";
		String filename, resourceName;
		
		resourceName = "org/NooLab/itexx/app/gui/resources/";
		resourceName = resourceName + "itexx google results ok-s.png";
		
		ResourceLoader rL = new ResourceLoader();
		
		if (rL.loadImageResource( this.getClass(),resourceName )){
			filename = rL.saveToTmpFile();
			if (filename.length()>0){
				imagesource = "<img src=\""+filename+"\"  />";
			}
		}
		
		// remove/replace div:  <div class="gbqfqw" id="gbqfqw">
		Element div = soupdoc.select("div#gbqfqw").first() ; // FIXME make this an external configuration !!! google may change its signatures 
		
		if (div !=null){
			div.replaceWith(new Element(Tag.valueOf("div"), ""));
		}
		
		xhtml = "<!--creator=itexx --><p style=\"margin-top:40px;margin-left:20px;font-size:1.6em;\">"+
			   "Please log into Google with your Google account from within <em>iTexx</em></p> " +
			   "<p style=\"margin-left:20px;font-size:1.2em;\">(You may probably need to enlarge the window) !</br>"+
			   "After logging in, you also have to configure Google's settings, such as never to display the instant results.</br>"+
			   "Else, it is recommended to set the size of the result list to 100 items.</p>"+
			   ""+
			   "</p>"+
			   "<p style=\"margin-top:40px;margin-left:20px;font-size:1.2em;\">"+
			   "You should see something like the following output,</br> i.e. a processed list:</br></br>"+
			   //"<img src=\"\"  />"+
			   imagesource+
			   /*
			    * actually, here we should load from resources write to disk (java temp)
			    * and display accordingly
			    */
			   "</p>";
			   // "</body></html>";
		
		html = soupdoc.html();
		html = html.replace("</BODY", "</body");
		int p = html.indexOf("<body");
		if (p>0){
			p = html.indexOf("<",p+2);
			html = strgutil.insert(html, xhtml, p);
			
			str = html.substring(p-5,p+200);
			String tmpFile = DFutils.getTempFile("googlehtml-3.txt");
			(new DFutils()).writeFileSimple( tmpFile, html) ;

			str = str +" ";
		}
		
		
		soupdoc = Jsoup.parse(html) ;
		html = soupdoc.html();
		
		return html;
	}
	
	private Document assemblingBareList(Document soupdoc, String allScripts) {
		
		Document sdoc = soupdoc;
		String html="";
		
		Element itemList = soupdoc.select("div#search").first();
		// "<div id=\"search\">"
		
		if (itemList!=null){
			html = itemList.outerHtml() ;
			
			if ((html!=null) && (html.length()>0)){
				html = "<html>\n"+allScripts+"\n\n<body id=\"gsr\">"+itemList+"</body></html>";
				sdoc = Jsoup.parse(html) ;
			}
		}// null ?
		
		
		return sdoc;
	}
	private String getAllScripts(Document soupdoc) {
		// 

		Elements scripts = soupdoc.select("script") ;
		String completeHead = "";
		String allScripts ="";

		for(int i=0;i<scripts.size();i++){
			allScripts = allScripts + scripts.get(i).outerHtml() ;
		}

		
		return allScripts ;
	}
	private String selectBody(Document soupdoc){

		String html="",bodyStr;
		
		bodyStr = (soupdoc.select( "body").get(0)).outerHtml() ;

		if (bodyStr.length()>70){
			html = "<html>"+bodyStr+"</html>";
		}
		return html;
	}
	
	private void removeScripts(Document soupdoc) {
	
		Elements scripts = soupdoc.select("script");
		if ((scripts!=null) && (scripts.size()>0)){
		
			for (int i=0;i<scripts.size();i++){
				scripts.get(i).replaceWith(new Element(Tag.valueOf("div"), ""));
			}
		}
	}
	
	public void removeSearchField(PageIntf page) {
		// <div id="gbx1" class="gbes">  <div id="gb" class="gbes">
		// google start page :  <div style="position: relative;" class="tsf-p">
		
		StringsUtil strgutil = new StringsUtil();
		
		String[] divClasses = new String[]{"gbes", "tsf-p"} ;
		// String[] divIds = new String[]{"gbx1", "gb", "gbx*"} ;
		// <div id="search">
		String[] divIds = new String[]{"search" } ;
		
		ArrayList<Integer> identifiedDivPositions = new ArrayList<Integer>();
		
		String htmlstr, ds;
		int p,pc,pn;
		int[] divpositions ; 
		
		htmlstr =  page.getHTML() ;
		
		if (htmlstr.contains("\"main\"")==false){
			return; // remove that
		}
		
		// (new DFutils()).writeFileSimple(DFutils.getTempFile("googlehtml.txt"), htmlstr) ;
		p = htmlstr.indexOf( " id=\""+divIds[0]) -3; 
		if (p<0){p=0;}
		divpositions = strgutil.indexesOfStrings(htmlstr, new String[]{"<div"}, p, true);
		
		for (int i=0;i<divpositions.length;i++){
			
			p = divpositions[i] ;
			pc = htmlstr.indexOf(">", p+1);
			pn = htmlstr.indexOf("<", p+2);
			
			if ((p>0) && (pc>p) && (pc<pn)){
				
				ds = htmlstr.substring(p,pc+1) ;
				if ((ds.length()>0)){
					if (strgutil.indexesOfStrings(ds, divIds, 0, true).length>0){
						identifiedDivPositions.add( p );
					}
					
				}
			}
		} // i->
		
		for (int i=0;i<identifiedDivPositions.size();i++){
			// not functional so far
			htmlstr = strgutil.removeTagfromHtml( htmlstr,"div", identifiedDivPositions.get(i));
		}
		
	}
	
	
	
}
